#!/bin/bash


#--------------------------CONFIGURATIONS-----------------------------

# Name of the executable
execroot="./../bin"
#executable="cg-dd"
executable="dg-comp"

# Number of cores
num_cores=1
# Number of MPI processes (per core)
num_MPI_proc=4

# Name specifier (used to identify different experiments)
name_spec="fermi-test"

# Input file
infile=${executable}.in

# Wtime
wtime="00:00:10"

#------------------------END CONFIGURATIONS---------------------------


#-----------------------------MAIN OUTPUT-----------------------------

# PBS script to submit the job
job_file="ll_script_to_run_${executable}.sub"

# job name
job_qual="${num_cores}x${num_MPI_proc}"
job_name="${executable}_${job_qual}"

# stdout files: the first one collects the scheduler output (including
# a copy of the present file), the second one is the output of the
# executable, redirected with ">"
job_out="out-${executable}-${name_spec}-N${job_qual}"
job_out2="${job_out}-ll.txt"
job_out1="${job_out}.txt"

#---------------------------END MAIN OUTPUT---------------------------


# We make a copy so that the original file can be changed even while
# the job is waiting in the queue.
input_file=""
if [ -n "${infile}" ]; # notice that we need "" in the test!
 then
  input_file="temp-${job_name}-${RANDOM}"
  cp ${infile} "${input_file}"
fi

# Total number of MPI processes
tot_num_MPI_proc=`echo ${num_cores} ${num_MPI_proc} | awk '{ N = \$1*\$2 ; print N }'`

# Write the LoadLeveler input file
cat > ${job_file} << EOF
#!/bin/bash
# @ job_type = bluegene
# @ wall_clock_limit = ${wtime}
# @ notification = never
# @ bg_size = ${num_cores}
# @ job_name = ${job_name}
# @ initialdir = .
# @ account_no = IscrC_NORDACS
# @ error  = ${job_out2}
# @ output = ${job_out2}
# @ queue

# echo the LL input file (this file)
cat ${job_file}
echo
echo
echo "#><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>#"
echo
echo

# load the right module
module load bgq-xl

# Run the executable
runjob --np ${tot_num_MPI_proc}          \\
       --ranks-per-node ${num_MPI_proc}  \\
       --env-all                         \\
       : ${execroot}/${executable} ${input_file} > ${job_out1}

date
EOF

# Avoid mixing LL output from different jobs
if [ -e "${job_out2}" ]
 then
  rm ${job_out2}
fi

llsubmit ${job_file}

